import org.json.JSONArray;
import org.json.JSONObject;

import java.io.*;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;

/**
 * Created by lucas on 2017/4/11.
 */
public class GlobseCrawler implements Runnable {

    private static final Logger LOG = Logger.getLogger(GlobseCrawler.class.getName());
    private static final String CNWORDFILE = "data/cnword.txt";
    private static final String GLOBSEPATH = "outCorpus/cnjp/globse/";
    private static int threadNum = 50;
    private static ProxyUtil proxyUtil;
    private static long lastTime = 0;
    private String word;
    private static String proxy;

    @Override
    public void run() {
        try {

            for (int page = 1; page < 8; page++) {
                BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(GLOBSEPATH + word + ".txt", true));
                String ret = getRet("https://glosbe.com/gapi/tm?from=zho&dest=jpn&format=json&phrase="
                        + URLEncoder.encode(word) + "&page=" + page);

                if (ret == null) {
                    continue;
                }
                JSONObject jsonObject = new JSONObject(ret);
                JSONArray examples = jsonObject.getJSONArray("examples");
                for (int i = 0; i < examples.length(); i++) {
                    JSONObject example = examples.getJSONObject(i);
                    bufferedWriter.write(example.getString("first") + "\t" + example.getString("second") + "\n");
                }
                bufferedWriter.close();
            }
            LOG.info(word + " done");

        } catch (Exception e) {
            LOG.info("get corpus error in " + word);
        }
    }

    public GlobseCrawler(String word) {
        this.word = word;
    }

    public static String getRet(String url) {

        String ret = null;
        try {
            Date d = new Date();
            long currentTime = d.getTime();
            //LOG.info(""+currentTime);
            //LOG.info(currentTime + " " + lastTime);
            if (proxy != null) {
                try {
                    ret = HttpRequestUtil.sendGetWithProxy(url, proxy.split("\t")[0], proxy.split("\t")[1]);
                    //Thread.sleep(200);
                    if (ret == null) {

                        proxy = proxyUtil.getProxyFromXdaili(url);
                        if (proxy == null) lastTime = currentTime;

                        if (proxy != null) {
                            ret = HttpRequestUtil.sendGetWithProxy(url, proxy.split("\t")[0], proxy.split("\t")[1]);
                            //Thread.sleep(200);
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            } else if (currentTime - lastTime > 2 * 60 * 1000) {
                LOG.info("exceeding 2 minutes, getting new proxy");
                proxy = proxyUtil.getProxyFromXdaili(url);
                if (proxy == null) lastTime = currentTime;
                if (proxy != null) {
                    ret = HttpRequestUtil.sendGetWithProxy(url, proxy.split("\t")[0], proxy.split("\t")[1]);
                    //Thread.sleep(200);
                }
            }
            if (ret == null) {
                ret = HttpRequestUtil.sendGet(url);
                LOG.info("using self ip");
                Thread.sleep(2000);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return ret;
    }


    public static void main(String args[]) {
        try {
            proxyUtil = new ProxyUtil();
            Set<String> set = new HashSet<String>();
            File dir = new File(GLOBSEPATH);
            File files[] = dir.listFiles();
            for (File file : files) {
                set.add(file.getName().replace(".txt", ""));
            }

            BufferedReader bufferedReader = new BufferedReader(new FileReader(CNWORDFILE));
            String word;
            ArrayList<String> arrayList = new ArrayList<String>();
            while ((word = bufferedReader.readLine()) != null) {
                if (set.contains(word)) {
                    arrayList.clear();
                    continue;
                }
                arrayList.add(word);
            }
            bufferedReader.close();
            ThreadPoolExecutor ex = (ThreadPoolExecutor) Executors.newFixedThreadPool(threadNum);
            LOG.info("remain words: " + arrayList.size());
            for (String word_remain : arrayList) {
                GlobseCrawler job = new GlobseCrawler(word_remain);
                ex.execute(job);
            }
            bufferedReader.close();
            ex.shutdown();
            while (true) {
                if (ex.awaitTermination(30, TimeUnit.SECONDS)) {
                    break;
                }
                LOG.info(ex.getActiveCount() + " fetcher currently active.");
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

    }
}
